{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reinforce算法中是以蒙特卡洛采样法来估计Q(state,action)的\n",
    "\n",
    "Actor_Critic算法中以神经网络来估算Q(state,action)\n",
    "\n",
    "再使用td误差来训练该神经网络"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:55:55.670467Z",
     "start_time": "2025-01-04T07:55:53.675074Z"
    }
   },
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUf0lEQVR4nO3dfWxT570H8K8dxw55OQ4hjd1cYoFUNhbx0jVAOO0fm1qPjEa9Y82kbeJ2GUJUMAeVZhdpkVoQ3aZUTFq7bhQ0qYNOU0uVXdGKXPoSBRq0iyEQFt0QSjSkdskabA9Y7LwQ27F/9w9uDnUIlITEjx1/P9KR8PM8Pv6dA/5yfJ7jY5OICIiIFDCrLoCIMhcDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlFEWQPv27cOiRYuQk5ODyspKtLe3qyqFiBRREkDvvPMO6uvrsXv3bpw/fx4rV65EVVUVAoGAinKISBGTii+jVlZWYvXq1fjd734HAIjH4ygrK8P27dvxs5/9LNnlEJEilmS/YCQSQUdHBxoaGow2s9kMt9sNr9c76XPC4TDC4bDxOB6P4/r161iwYAFMJtOs10xEUyMiGBwcRGlpKczmO3/QSnoAXb16FbFYDA6HI6Hd4XDg0qVLkz6nsbERe/bsSUZ5RDSD+vr6sHDhwjv2Jz2ApqOhoQH19fXG42AwCJfLhb6+PmiaprAyIppMKBRCWVkZCgoK7jou6QFUXFyMrKws+P3+hHa/3w+n0znpc2w2G2w2223tmqYxgIhS2JedIkn6LJjVakVFRQVaW1uNtng8jtbWVui6nuxyiEghJR/B6uvrUVtbi1WrVmHNmjV49dVXMTw8jE2bNqkoh4gUURJA3//+9/HPf/4Tu3btgs/nw8MPP4wPPvjgthPTRDS3KbkO6H6FQiHY7XYEg0GeAyJKQff6HuV3wYhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKTMlAPo5MmTeOqpp1BaWgqTyYR33303oV9EsGvXLjz44IOYN28e3G43/va3vyWMuX79OjZu3AhN01BYWIjNmzdjaGjovjaEiNLPlANoeHgYK1euxL59+ybt37t3L1577TUcOHAAZ86cQV5eHqqqqjA6OmqM2bhxI7q7u9HS0oLm5macPHkSzz777PS3gojSk9wHAHLkyBHjcTweF6fTKb/61a+MtoGBAbHZbPL222+LiMjFixcFgJw9e9YY8/7774vJZJLPP//8nl43GAwKAAkGg/dTPhHNknt9j87oOaBPP/0UPp8PbrfbaLPb7aisrITX6wUAeL1eFBYWYtWqVcYYt9sNs9mMM2fOTLrecDiMUCiUsBBR+pvRAPL5fAAAh8OR0O5wOIw+n8+HkpKShH6LxYKioiJjzESNjY2w2+3GUlZWNpNlE5EiaTEL1tDQgGAwaCx9fX2qSyKiGTCjAeR0OgEAfr8/od3v9xt9TqcTgUAgoX9sbAzXr183xkxks9mgaVrCQkTpb0YDaPHixXA6nWhtbTXaQqEQzpw5A13XAQC6rmNgYAAdHR3GmOPHjyMej6OysnImyyGiFGeZ6hOGhoZw+fJl4/Gnn36Kzs5OFBUVweVyYceOHfjFL36BJUuWYPHixXjxxRdRWlqKDRs2AAC+9rWv4dvf/ja2bNmCAwcOIBqNoq6uDj/4wQ9QWlo6YxtGRGlgqtNrJ06cEAC3LbW1tSJycyr+xRdfFIfDITabTZ544gnp6elJWMe1a9fkhz/8oeTn54umabJp0yYZHByc8Sk+IlLjXt+jJhERhfk3LaFQCHa7HcFgkOeDiFLQvb5H02IWjIjmJgYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpMyUf5aHaDaICAb7exAdCRptOfMfRF6xS2FVNNsYQJQaJI7+8/+Nwf4eo8m5ch0DaI7jRzBKCRKPIR6Lqi6DkowBRClB4nHIGAMo0zCAKCXEY1HExiIJbSYzzxDMdQwgSgmx8DCiw/+61WAyIXfBQnUFUVIwgCglCG7OhN1igjk7R1U5lCQMIEpZWdk21SXQLJtSADU2NmL16tUoKChASUkJNmzYgJ6enoQxo6Oj8Hg8WLBgAfLz81FTUwO/358wpre3F9XV1cjNzUVJSQl27tyJsbGx+98amjNMJhPMFgbQXDelAGpra4PH48Hp06fR0tKCaDSKdevWYXh42Bjz/PPP4+jRo2hqakJbWxv6+/vx9NNPG/2xWAzV1dWIRCI4deoU3nzzTRw6dAi7du2aua2itCOxMdz8IHaLycwD9DlP7kMgEBAA0tbWJiIiAwMDkp2dLU1NTcaYTz75RACI1+sVEZFjx46J2WwWn89njNm/f79omibhcPieXjcYDAoACQaD91M+pZDBK5fl7O+3SfuBLdJ+YIuc/f1WGb76D9Vl0TTd63v0vv6LCQZvXjZfVFQEAOjo6EA0GoXb7TbGLF26FC6XC16vFwDg9XqxfPlyOBwOY0xVVRVCoRC6u7snfZ1wOIxQKJSw0NwSGwtj4hEQzX3TDqB4PI4dO3bgsccew7JlywAAPp8PVqsVhYWFCWMdDgd8Pp8x5ovhM94/3jeZxsZG2O12YykrK5tu2ZSiYpHRxFkwkwkmk7p6KDmmHUAejwcXLlzA4cOHZ7KeSTU0NCAYDBpLX1/frL8mJdfov/oBiRuPrbmFsOQUKKyIkmFal5rW1dWhubkZJ0+exMKFty4WczqdiEQiGBgYSDgK8vv9cDqdxpj29vaE9Y3Pko2Pmchms8Fm44zIXBaPxxIemy02mCzZiqqhZJnSEZCIoK6uDkeOHMHx48exePHihP6KigpkZ2ejtbXVaOvp6UFvby90XQcA6LqOrq4uBAIBY0xLSws0TUN5efn9bAvNIaYsC8zmLNVl0Cyb0hGQx+PBW2+9hffeew8FBQXGORu73Y558+bBbrdj8+bNqK+vR1FRETRNw/bt26HrOtauXQsAWLduHcrLy/HMM89g79698Pl8eOGFF+DxeHiUk6ESzv38P3OWBSYG0Jw3pQDav38/AOCb3/xmQvvBgwfx4x//GADwyiuvwGw2o6amBuFwGFVVVXj99deNsVlZWWhubsa2bdug6zry8vJQW1uLl1566f62hNKaTPgIBpP55kJzmkkm++8nxYVCIdjtdgSDQWiaprocuk8icfT+zzsIdJ8w2vKdS7D03/8TJk6FpaV7fY/yvxhST4D4WFh1FaQAA4hSgCAWnRBAPPDJCAwgUk7iMYwOJF6Emlv0b4qqoWRiAJFyIoL4hLsh8iLEzMAAopTEewFlBgYQpSTeDTEzMIBIPYnfdjGiOYsXIWYCBhApF4+NARMvRISJ1wBlAAYQKRePRSFf+CY8ZQ4GECkXj4ZvHgV9AW/Hmhn4t0zKRYauIRa+dV9xs8UGm1aisCJKFgYQKTfxBLTJbIaZ0/AZgQFEqcdkhjmLNyPLBAwgSjk3fxPMqroMSgIGECl3+72ATLwZWYZgAJFy8YnfhKeMwQAi5W67FQdlDAYQKReL3Eh4bDKZwR8FywwMIFJu5Fri77zZtBLOgmUIBhApJxOugs6y5vAkdIZgAFHKMVus/CJqhmAAUcoxW6w8B5QhGECklIjc9k14c5YFvCt9ZmAAkVIiccRj0dva+REsM0zpl1GJpiMcDuPGjRuT9kksimj4xoTxEQwMDNxxfbm5ubBa+VWNuYABRLPuj3/8I/bs2TNpX441C3v+oxJLSguNtqamJhzYtPuO63v11Vfxve99b6bLJAUYQDTrhoaG8Pnnn0/aV2zPxfz8PFy+8QiuR0vxQPbf8b+Xz9xxPACMjIzMVqmUZFM6B7R//36sWLECmqZB0zTouo7333/f6B8dHYXH48GCBQuQn5+Pmpoa+P3+hHX09vaiuroaubm5KCkpwc6dOzE2NjbxpShDZJlN+MdYBS6P3AygnpE1CFkqVJdFSTKlAFq4cCFefvlldHR04Ny5c3j88cfxne98B93d3QCA559/HkePHkVTUxPa2trQ39+Pp59+2nh+LBZDdXU1IpEITp06hTfffBOHDh3Crl27ZnarKK0Mxebj1j/FLFhznSrLoSSaUgA99dRTePLJJ7FkyRJ85StfwS9/+Uvk5+fj9OnTCAaDeOONN/DrX/8ajz/+OCoqKnDw4EGcOnUKp0+fBgB89NFHuHjxIv70pz/h4Ycfxvr16/Hzn/8c+/btQyQS+ZJXp7nqQdtlWExhAIJsjKD/76dUl0RJMu1zQLFYDE1NTRgeHoau6+jo6EA0GoXb7TbGLF26FC6XC16vF2vXroXX68Xy5cvhcDiMMVVVVdi2bRu6u7vx9a9/fUo1XLp0Cfn5+dPdBEqSiR/Dv2joRgRHP/ovDMspDIyVYL7lCno/O3vX9fX39+PixYszXSbNoKGhoXsaN+UA6urqgq7rGB0dRX5+Po4cOYLy8nJ0dnbCarWisLAwYbzD4YDP5wMA+Hy+hPAZ7x/vu5NwOIxw+NYtG0KhEAAgGAzy/FEauNMUPAAMj0bxdmsXgK57Xt/IyMhdp+lJveHh4S8fhGkE0Fe/+lV0dnYiGAziz3/+M2pra9HW1jblAqeisbFx0mncyspKaJo2q69N9+/MmTMzur6HHnoIjz766Iyuk2bW+EHCl5nyldBWqxUPPfQQKioq0NjYiJUrV+I3v/kNnE4nIpHbLyDz+/1wOm+eVHQ6nbcdjo8/Hh8zmYaGBgSDQWPp6+u741giSh/3/VWMeDyOcDiMiooKZGdno7W11ejr6elBb28vdF0HAOi6jq6uLgQCAWNMS0sLNE1DeXn5HV/DZrMZU//jCxGlvyl9BGtoaMD69evhcrkwODiIt956Cx9//DE+/PBD2O12bN68GfX19SgqKoKmadi+fTt0XcfatWsBAOvWrUN5eTmeeeYZ7N27Fz6fDy+88AI8Hg9sNv4OFFGmmVIABQIB/OhHP8KVK1dgt9uxYsUKfPjhh/jWt74FAHjllVdgNptRU1ODcDiMqqoqvP7668bzs7Ky0NzcjG3btkHXdeTl5aG2thYvvfTSzG4VpZTxI9iZwu+BzR0mmfizlGkgFArBbrcjGAzy41gaGBwcnNFZq6KiIuTl5c3Y+mjm3et7lN8Fo1lXUFCAgoIC1WVQCuL9gIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyFtUFTIeIAABCoZDiSohoMuPvzfH36p2kZQBdu3YNAFBWVqa4EiK6m8HBQdjt9jv2p2UAFRUVAQB6e3vvunGUKBQKoaysDH19fdA0TXU5aYH7bHpEBIODgygtLb3ruLQMILP55qkru93OfxTToGka99sUcZ9N3b0cHPAkNBEpwwAiImXSMoBsNht2794Nm82mupS0wv02ddxns8skXzZPRkQ0S9LyCIiI5gYGEBEpwwAiImUYQESkTFoG0L59+7Bo0SLk5OSgsrIS7e3tqktSprGxEatXr0ZBQQFKSkqwYcMG9PT0JIwZHR2Fx+PBggULkJ+fj5qaGvj9/oQxvb29qK6uRm5uLkpKSrBz506MjY0lc1OUefnll2EymbBjxw6jjfssSSTNHD58WKxWq/zhD3+Q7u5u2bJlixQWForf71ddmhJVVVVy8OBBuXDhgnR2dsqTTz4pLpdLhoaGjDFbt26VsrIyaW1tlXPnzsnatWvl0UcfNfrHxsZk2bJl4na75a9//ascO3ZMiouLpaGhQcUmJVV7e7ssWrRIVqxYIc8995zRzn2WHGkXQGvWrBGPx2M8jsViUlpaKo2NjQqrSh2BQEAASFtbm4iIDAwMSHZ2tjQ1NRljPvnkEwEgXq9XRESOHTsmZrNZfD6fMWb//v2iaZqEw+HkbkASDQ4OypIlS6SlpUW+8Y1vGAHEfZY8afURLBKJoKOjA26322gzm81wu93wer0KK0sdwWAQwK0v7HZ0dCAajSbss6VLl8Llchn7zOv1Yvny5XA4HMaYqqoqhEIhdHd3J7H65PJ4PKiurk7YNwD3WTKl1ZdRr169ilgslvCXDgAOhwOXLl1SVFXqiMfj2LFjBx577DEsW7YMAODz+WC1WlFYWJgw1uFwwOfzGWMm26fjfXPR4cOHcf78eZw9e/a2Pu6z5EmrAKK783g8uHDhAv7yl7+oLiWl9fX14bnnnkNLSwtycnJUl5PR0uojWHFxMbKysm6bjfD7/XA6nYqqSg11dXVobm7GiRMnsHDhQqPd6XQiEolgYGAgYfwX95nT6Zx0n473zTUdHR0IBAJ45JFHYLFYYLFY0NbWhtdeew0WiwUOh4P7LEnSKoCsVisqKirQ2tpqtMXjcbS2tkLXdYWVqSMiqKurw5EjR3D8+HEsXrw4ob+iogLZ2dkJ+6ynpwe9vb3GPtN1HV1dXQgEAsaYlpYWaJqG8vLy5GxIEj3xxBPo6upCZ2ensaxatQobN240/sx9liSqz4JP1eHDh8Vms8mhQ4fk4sWL8uyzz0phYWHCbEQm2bZtm9jtdvn444/lypUrxjIyMmKM2bp1q7hcLjl+/LicO3dOdF0XXdeN/vEp5XXr1klnZ6d88MEH8sADD2TUlPIXZ8FEuM+SJe0CSETkt7/9rbhcLrFarbJmzRo5ffq06pKUATDpcvDgQWPMjRs35Cc/+YnMnz9fcnNz5bvf/a5cuXIlYT2fffaZrF+/XubNmyfFxcXy05/+VKLRaJK3Rp2JAcR9lhy8HQcRKZNW54CIaG5hABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMr8H3ZrxgaPOuNuAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:56:24.190423Z",
     "start_time": "2025-01-04T07:55:55.681936Z"
    }
   },
   "source": [
    "import torch\n",
    "\n",
    "#演员模型,计算每个动作的概率\n",
    "model_actor = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "'''评委模型：双模型组合：一个计算估计值value,另一个计算目标值target'''\n",
    "#评委模型,计算每个状态的价值\n",
    "model_critic = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "\n",
    "model_critic_delay = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "# 将当前 Critic 网络（model_critic）的参数复制到一个延迟更新的 Critic 网络（model_critic_delay）中。\n",
    "# 注意是将括号内部的参数复制给括号外部的\n",
    "model_critic_delay.load_state_dict(model_critic.state_dict())\n",
    "\n",
    "model_actor, model_critic"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       "   (5): Softmax(dim=1)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=1, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:56:24.502165Z",
     "start_time": "2025-01-04T07:56:24.449987Z"
    }
   },
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    state = []\n",
    "    action = []\n",
    "    reward = []\n",
    "    next_state = []\n",
    "    over = []\n",
    "\n",
    "    s = env.reset()\n",
    "    o = False\n",
    "    while not o:\n",
    "        #根据概率采样\n",
    "        prob = model_actor(torch.FloatTensor(s).reshape(1, 4))[0].tolist()\n",
    "        a = random.choices(range(2), weights=prob, k=1)[0]\n",
    "\n",
    "        ns, r, o = env.step(a)\n",
    "\n",
    "        state.append(s)\n",
    "        action.append(a)\n",
    "        reward.append(r)\n",
    "        next_state.append(ns)\n",
    "        over.append(o)\n",
    "\n",
    "        s = ns\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    state = torch.FloatTensor(state).reshape(-1, 4)\n",
    "    action = torch.LongTensor(action).reshape(-1, 1)\n",
    "    reward = torch.FloatTensor(reward).reshape(-1, 1)\n",
    "    next_state = torch.FloatTensor(next_state).reshape(-1, 4)\n",
    "    over = torch.LongTensor(over).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over, reward.sum().item()\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over, reward_sum = play()\n",
    "\n",
    "reward_sum"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\app\\anaconda3\\install\\envs\\pytorch_env_3_8\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n",
      "C:\\Users\\25036\\AppData\\Local\\Temp\\ipykernel_51928\\3214972205.py:34: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\torch\\csrc\\utils\\tensor_new.cpp:281.)\n",
      "  state = torch.FloatTensor(state).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-979.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:56:25.445005Z",
     "start_time": "2025-01-04T07:56:24.651808Z"
    }
   },
   "source": [
    "# 定义两个优化器：\n",
    "# - optimizer_actor：用于优化 Actor 网络（策略网络）的参数\n",
    "# - optimizer_critic：用于优化 Critic 网络（值函数网络）的参数\n",
    "optimizer_actor = torch.optim.Adam(model_actor.parameters(), lr=4e-3)  # Actor 优化器，学习率为 4e-3\n",
    "optimizer_critic = torch.optim.Adam(model_critic.parameters(), lr=4e-2)  # Critic 优化器，学习率为 4e-2\n",
    "\n",
    "# 定义一个函数，用于设置模型参数是否需要计算梯度\n",
    "def requires_grad(model, value):\n",
    "    for param in model.parameters():  # 遍历模型中所有的参数\n",
    "        param.requires_grad_(value)  # 设置参数的 requires_grad 属性为指定的 value（True 或 False）\n"
   ],
   "outputs": [],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T08:11:59.608083Z",
     "start_time": "2025-01-04T08:11:59.600441Z"
    }
   },
   "source": [
    "# 定义一个函数，用于训练 Critic 网络\n",
    "def train_critic(state, reward, next_state, over):\n",
    "    # 冻结 Actor 网络（不计算梯度，不更新参数），Actor 网络的参数不需要在 Critic 网络更新时被修改，因此冻结 Actor 网络以节省计算资源。\n",
    "    requires_grad(model_actor, False)\n",
    "    # 启用 Critic 网络的梯度计算（允许更新参数），启用 Critic 网络的参数梯度计算，以便在训练过程中更新它的参数。\n",
    "    requires_grad(model_critic, True)\n",
    "\n",
    "    # 1. **计算当前状态的值函数（Value）：**\n",
    "    # 使用 Critic 主网络计算当前状态的值函数估计\n",
    "    value = model_critic(state)\n",
    "\n",
    "    # 2. **计算目标值（Target）：**\n",
    "    # 使用 Critic 延迟网络（model_critic_delay）估计下一状态的值函数\n",
    "    # torch.no_grad() 确保在计算目标值时不进行梯度计算（节省资源）\n",
    "    with torch.no_grad(): #这个是pytorch中用于关闭梯度计算的操作，梯度计算是为了更新网络参数，这里model_critic_delay并不需要更新网络参数，delay 的网络参数是从主网络里复制来的，所以这里要关闭梯度计算\n",
    "        target = model_critic_delay(next_state)\n",
    "    \n",
    "    # 根据目标值公式更新：\n",
    "    # target = reward + γ * V(next_state) * (1 - over)\n",
    "    # - `0.99` 是折扣因子 γ，用于降低未来奖励的重要性\n",
    "    # - `(1 - over)` 是终止条件，表示如果游戏结束（over=1），未来值设为0\n",
    "    target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "    # 3. **计算时序差分（TD）误差：**\n",
    "    # Critic 的目标是使 value 逼近 target，使用均方误差 (MSE) 作为损失函数\n",
    "    loss = torch.nn.functional.mse_loss(value, target)\n",
    "\n",
    "    # 4. **更新 Critic 网络：**\n",
    "    # 反向传播，计算梯度\n",
    "    loss.backward()\n",
    "    # 使用优化器更新 Critic 网络的参数\n",
    "    optimizer_critic.step()\n",
    "    # 清空优化器的梯度\n",
    "    optimizer_critic.zero_grad()\n",
    "\n",
    "    # 返回 value 的无梯度版本（detach 分离张量，避免计算图记录梯度）\n",
    "    # 使用 detach() 方法分离张量，返回不带梯度的值函数 \n",
    "    # 这样可以防止 value 参与后续的梯度计算，减少内存占用。\n",
    "    # 返回value，是为了在actor的训练中使用\n",
    "    return value.detach()\n",
    "\n",
    "# 调用 train_critic 函数，输入当前状态、奖励、下一状态和游戏终止标志\n",
    "value = train_critic(state, reward, next_state, over)\n",
    "\n",
    "# 输出 value 的形状（通常为 (batch_size, 1)）\n",
    "value.shape\n"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([22, 1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:56:25.625063Z",
     "start_time": "2025-01-04T07:56:25.602431Z"
    }
   },
   "source": [
    "'''actor：使用策略梯度算法实现'''\n",
    "\n",
    "def train_actor(state, action, value):\n",
    "    requires_grad(model_actor, True)\n",
    "    requires_grad(model_critic, False)\n",
    "\n",
    "    #重新计算动作的概率\n",
    "    prob = model_actor(state)\n",
    "    prob = prob.gather(dim=1, index=action)\n",
    "\n",
    "    #根据策略梯度算法的导函数实现\n",
    "    #函数中的Q(state,action),这里使用critic模型估算\n",
    "    # 这里的value不再像传统的策略梯度那样使用蒙特采样法估算，而是来自于critic的计算\n",
    "    prob = (prob + 1e-8).log() * value\n",
    "    loss = -prob.mean()\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer_actor.step()\n",
    "    optimizer_actor.zero_grad()\n",
    "\n",
    "    return loss.item()\n",
    "\n",
    "\n",
    "train_actor(state, action, value)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.005793537013232708"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-04T07:56:45.769550Z",
     "start_time": "2025-01-04T07:56:25.645046Z"
    }
   },
   "source": [
    "def train():\n",
    "    # 设置 Actor 和 Critic 网络为训练模式（启用 dropout 或其他正则化机制）\n",
    "    model_actor.train()\n",
    "    model_critic.train()\n",
    "\n",
    "    # 训练 1000 个 epoch\n",
    "    for epoch in range(1000):\n",
    "\n",
    "        # 一个 epoch 至少与环境交互 200 步\n",
    "        steps = 0\n",
    "        while steps < 200:\n",
    "            # 调用 play() 函数采样轨迹，获取以下数据：\n",
    "            # state: 当前状态\n",
    "            # action: 选择的动作\n",
    "            # reward: 即时奖励\n",
    "            # next_state: 下一状态\n",
    "            # over: 是否结束标志（1 表示终止，0 表示继续）\n",
    "            # _: 可忽略的额外信息（可能是累计奖励或调试信息）\n",
    "            state, action, reward, next_state, over, _ = play()\n",
    "\n",
    "            # 增加步数统计\n",
    "            steps += len(state)\n",
    "\n",
    "            # 更新 Critic 网络，训练值函数 V(s)，返回当前值函数估计\n",
    "            value = train_critic(state, reward, next_state, over)\n",
    "\n",
    "            # 更新 Actor 网络，优化策略，返回 Actor 的损失\n",
    "            loss = train_actor(state, action, value)\n",
    "\n",
    "        # 每个 epoch 结束后，更新延迟网络（model_critic_delay）\n",
    "        # 按比例（0.7 对 0.3）对 Critic 主网络和延迟网络的参数进行平滑更新\n",
    "        for param, param_delay in zip(model_critic.parameters(),\n",
    "                                      model_critic_delay.parameters()):\n",
    "            # param: Critic 主网络的参数\n",
    "            # param_delay: 延迟网络的参数\n",
    "            value = param_delay.data * 0.7 + param.data * 0.3  # 权重更新\n",
    "            param_delay.data.copy_(value)  # 将新值复制到延迟网络的参数中\n",
    "\n",
    "        # 每隔 100 个 epoch 进行一次测试\n",
    "        if epoch % 100 == 0:\n",
    "            # 使用当前策略运行 20 局游戏，统计平均奖励\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "\n",
    "            # 打印当前 epoch、Actor 网络损失和测试结果\n",
    "            print(epoch, loss, test_result)\n",
    "\n",
    "# 调用 train() 函数，开始训练 Actor 和 Critic 网络\n",
    "train()\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -11.128024101257324 -973.7\n",
      "100 -392.2756652832031 -949.45\n",
      "200 -216.72608947753906 -586.35\n",
      "300 -122.15315246582031 -160.25\n",
      "400 -78.09835815429688 -439.9\n",
      "500 -77.3221206665039 97.0\n",
      "600 -42.8882942199707 200.0\n",
      "700 -20.385530471801758 200.0\n",
      "800 -44.55078125 -216.25\n",
      "900 -8.7194242477417 200.0\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "play(True)[-1]"
   ],
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
